package com.lxj.xianyu;

import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.scheduler.BloomFilterDuplicateRemover;
import us.codecraft.webmagic.scheduler.QueueScheduler;

import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

@Service
public class SpiderService {

    ExecutorService executorService = Executors.newCachedThreadPool();

    @Autowired
    SpringDataPipeLine springDataPipeLine;

    @Autowired
    StringRedisTemplate stringRedisTemplate;

    private static final String path = "https://s.2.taobao.com/list/?_input_charset=utf8&st_edtime=1&q=%s&ist=1&page=%s";

    public void execute() {
        Set<String> keywords = stringRedisTemplate.opsForSet().members("keywords");
        if (!CollectionUtils.isEmpty(keywords)) {
            for (String keyword : keywords) {
                executorService.execute(() -> {
                    this.execute(keyword);
                });
            }
        }
    }

    public void execute(String keyword) {
        Spider spider = Spider.create(new JobProcessor(keyword)).addUrl(String.format(path, keyword, "1"));
        spider.setScheduler(new QueueScheduler().setDuplicateRemover(new BloomFilterDuplicateRemover(100000)));
        spider.thread(10);
        spider.addPipeline(springDataPipeLine);

        spider.run();
    }
}
